const phantom = require('phantom');
const cheerio = require('cheerio');
const fs = require("fs");
const csv = require('fast-csv');


// 爬取页面地址,第一页
const URL_PREFIX = "https://list.jd.com/list.html?cat=737,794,878&page="
const URL_SUFFIX = "&stock=0&sort=sort_rank_asc&trans=1&JL=4_7_0#J_main";

const Pa = async() => {
    const instance_getnum = await phantom.create(['--ignore-ssl-errors=yes', '--load-images=no']);
    const page_getnum = await instance_getnum.createPage();
    await page_getnum.open(URL_PREFIX + "1" + URL_SUFFIX);
    const listhref = [];
    //禁用详细页面的javascript以提高速度，所以以下三个内容必须从列表页读取了，故定义数组保存
    const list价格 = [];
    const list累计评价 = [];
    const list选购指数 = [];

    var csvStream = csv
        .createWriteStream({
            headers: true
        })
        .transform(function(row) {
            return {
                //基本信息
                名称: row.a,
                价格: row.b,
                累计评价: row.c,
                选购指数: row.d,
                //主体
                品牌: row.e,
                型号: row.f,
                颜色: row.g,
                类别: row.h,
                开门方式: row.i,
                显示方式: row.j,
                气候类型: row.k,
                // 功能
                制冷方式: row.l,
                控制方式: row.m,
                制冷类型: row.n,
                除霜模式: row.o,
                "定频/变频": row.p,
                "总容积(升)": row.q,
                "冷冻室(升)": row.r,
                "冷藏室(升)": row.s,
                "变温室(升)": row.t,
                "冷冻能力(kg/24h)": row.u,
                制冷循环: row.v,
                "耗电量(KWh/24h)": row.w,
                能效等级: row.x,
                "运转音dB(A)": row.y,
                制冷剂: row.z,
                // 规格
                "电压/频率": row.aa,
                "产品尺寸（深x宽x高）mm": row.ab,
                "包装尺寸（深x宽x高）mm": row.ac,
                "产品重量（kg）": row.ad,
                // 特性
                特性: row.ae
            };
        }),
        writableStream = fs.createWriteStream("京东冰箱信息.csv");

    writableStream.on("finish", function() {
        console.log("全部数据抓取完毕！");
    });
    await csvStream.pipe(writableStream);
    await page_getnum.on('onResourceRequested', true, function(requestData) {});
    const content = await page_getnum.property('content');
    const tmp$ = cheerio.load(content);
    var pagenumhtml = tmp$('.p-skip').html();
    pagenumhtml = cheerio.load(pagenumhtml);
    var PAGETOTALNUM = pagenumhtml('b').text();
    await page_getnum.close();
    await instance_getnum.exit();
    var page, pageitem;
    for (var l = 70; l < PAGETOTALNUM; l++) {
        console.log("开始读取第" + (l + 1) + "页列表信息")
        const instance = await phantom.create(['--ignore-ssl-errors=yes', '--load-images=no']);
        page = await instance.createPage();
        var status = await page.open(URL_PREFIX + (l + 1) + URL_SUFFIX);
        while (status != "success") {
            await pageitem.close();
            console.log("网页暂时打不开，重试中..")
            status = await page.open(URL_PREFIX + (l + 1) + URL_SUFFIX);
        }
        const content = await page.property('content');
        const $ = await cheerio.load(content);
        //找所有的list
        await $('.gl-item').each(function(i, elem) {
            const li = cheerio.load($(this).html());
            listhref[i] = "http:" + li(".comment").attr("href");
            if (li('.js_ys').html() != null) {
                list价格[i] = li('.js_ys').text().replace("¥", "");
            }
            if (li('.comment').html() != null) {
                list累计评价[i] = li('.comment').text();
            }
            if (li('.buy-score').html() != null) {
                list选购指数[i] = li('.buy-score').text().replace("选购指数", "");
            }
        });
        console.log("读取第" + (l + 1) + "页列表信息成功")
            //循环读取每一个具体的网页循环读取每一个具体的网页
        for (var i = 0; i < listhref.length; i++) {
            var Ptable_item = [];
            var finaldata = {
                名称: "",
                价格: "",
                累计评价: "",
                选购指数: "",
                //主体
                品牌: "",
                型号: "",
                颜色: "",
                类别: "",
                开门方式: "",
                显示方式: "",
                气候类型: "",
                // 功能
                制冷方式: "",
                控制方式: "",
                制冷类型: "",
                除霜模式: "",
                "定频/变频": "",
                "总容积(升)": "",
                "冷冻室(升)": "",
                "冷藏室(升)": "",
                "变温室(升)": "",
                "冷冻能力(kg/24h)": "",
                制冷循环: "",
                "耗电量(KWh/24h)": "",
                能效等级: "",
                "运转音dB(A)": "",
                制冷剂: "",
                // 规格
                "电压/频率": "",
                "产品尺寸（深x宽x高）mm": "",
                "包装尺寸（深x宽x高）mm": "",
                "产品重量（kg）": "",
                // 特性
                特性: ""
            }
            pageitem = await instance.createPage();
            pageitem.setting('javascriptEnabled', "false");
            status = await pageitem.open(listhref[i]);
            console.log("开始爬取：", listhref[i])

            while (status != "success") {
                status = await pageitem.open(listhref[i]);
                await pageitem.close();
                console.log("网页暂时打不开，重试中..")
                status = await pageitem.open(listhref[i]);
            }

            await pageitem.on('onResourceRequested', true, function(requestData) {});
            const dcontent = await pageitem.property('content');
            detailcontent = await cheerio.load(dcontent);
            await pageitem.close();
            if (detailcontent(".ellipsis").html() != null) {
                finaldata.名称 = detailcontent(".ellipsis").text();
                finaldata.价格 = list价格[i];
                finaldata.累计评价 = list累计评价[i];
                finaldata.选购指数 = list选购指数[i];

                //会有没有详细信息的情况
                if (detailcontent("div[class=Ptable]").html() != null) {
                    //找到 规格与包装下的有内容
                    detailcontent = cheerio.load(detailcontent("div[class=Ptable]").html());
                    //检出 主体、功能、规格、特性四个内容，有时候特性是没有的，这样用循环读取，下一步可能为3也可能为4
                    await detailcontent('div[class=Ptable-item]').each(function(deatili, elem) {
                        Ptable_item[deatili] = $(this).html();
                    });
                    console.log("读取基本数据成功");
                    // 循环处理三（四）个内容，按dt<->dd读取，希望京东的dt是统一的
                    for (var k = 0; k < Ptable_item.length; k++) {
                        detailcontent = cheerio.load(Ptable_item[k]);
                        detailcontent(".Ptable-tips").remove();
                        detailcontent("h3").remove();
                        var datalistdd = [];
                        var datalistdt = [];

                        await detailcontent("dd").each(function(im, elem) {
                            datalistdd[im] = detailcontent(this).text();
                        });
                        await detailcontent("dt").each(function(im, elem) {
                            datalistdt[im] = detailcontent(this).text();
                        });
                        for (var im = 0; im < datalistdd.length; im++) {
                            finaldata[datalistdt[im]] = datalistdd[im];
                        }
                    }
                }

            } else { //全球购不太一样
                finaldata.名称 = detailcontent(".parameter2").children().first().text()
                finaldata.价格 = list价格[i];
                finaldata.累计评价 = list累计评价[i];
                finaldata.选购指数 = list选购指数[i];

                var Ptable_item = [];
                await detailcontent("th").remove();
                var tmp = [];
                detailcontent(".Ptable").find('td').each(function(i, elem) {
                    tmp[i] = detailcontent(this).text();
                });
                for (var d = 0; d < tmp.length; d += 2) {
                    finaldata[tmp[d]] = tmp[d + 1];
                }
            }
            await csvStream.write({
                a: finaldata['名称'],
                b: finaldata['价格'],
                c: finaldata['累计评价'],
                d: finaldata['选购指数'],
                e: finaldata['品牌'],
                f: finaldata['型号'],
                g: finaldata['颜色'],
                h: finaldata['类别'],
                i: finaldata['开门方式'],
                j: finaldata['显示方式'],
                k: finaldata['气候类型'],
                l: finaldata['制冷方式'],
                m: finaldata['控制方式'],
                n: finaldata['制冷类型'],
                o: finaldata['除霜模式'],
                p: finaldata['定频/变频'],
                q: finaldata['总容积(升)'],
                r: finaldata['冷冻室(升)'],
                s: finaldata['冷藏室(升)'],
                t: finaldata['变温室(升)'],
                u: finaldata['冷冻能力(kg/24h)'],
                v: finaldata['制冷循环'],
                w: finaldata['耗电量(KWh/24h)'],
                x: finaldata['能效等级'],
                y: finaldata['运转音dB(A)'],
                z: finaldata['制冷剂'],
                aa: finaldata['电压/频率'],
                ab: finaldata['产品尺寸（深x宽x高）mm'],
                ac: finaldata['包装尺寸（深x宽x高）mm'],
                ad: finaldata['产品重量（kg）'],
                ae: finaldata['特性'],
            });
            await console.log("写入第" + (l * 60 + i + 1) + "条数据");
        }
        await instance.exit();
    }
    await csvStream.end();
}

Pa();